# -*- coding: utf-8 -*-
import json
import scrapy


class KuaidailiSpider(scrapy.Spider):
    name = "kuaidaili"
    domain = 'http://www.kuaidaili.com'
    allowed_domains = ["www.kuaidaili.com"]
    start_urls = ['http://www.kuaidaili.com/']
    output = {'arr': []}
    arr_len = 0

    def parse(self, response):
        pages = response.css('div#listnav a::attr(href)').extract()
        self.arr_len = len(pages)
        for url in pages:
            yield scrapy.Request(self.domain + url, callback=self.parse_content)

    def parse_content(self, response):
        self.arr_len = self.arr_len - 1
        for tr in response.css('table tbody tr'):
            ip = tr.css('td[data-title="IP"]::text').extract_first()
            port = tr.css('td[data-title="PORT"]::text').extract_first()
            item = {'ip_port': ip + ':' + port, 'user_pass': ''}
            self.output['arr'].append(item)

        if self.arr_len == 0:
            fo = open("proxy_list.txt", "w")
            data = json.dumps(self.output)
            fo.write( data )
            fo.close()
